import re
import codecs
import os, sys

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BASE_DIR)
from utils_package.config.config import Config
cur = os.path.dirname(os.path.dirname(__file__))
config = Config(config_file=os.path.join(cur, "utils_package/config/conf.json"))


class LoadData():

    def __init__(self):
        self.punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~【】：，②"""

    def clean_sen(self, content):
        # 字母大小写转化
        content = content.lower()
        # 去英文标点符号
        content = re.sub(r'[%s]+' % self.punctuation, '', content)
        # 去中文标点符号
        content = re.sub(r'[%s]+' % self.punctuation, '', content)
        # 去换行符
        content = content.replace('\n', '')
        # 去空格
        content = re.sub(r'\s+', '', content)

        return content

    def get_stop_words(self):
        stopkey = [w.strip() for w in codecs.open(os.path.join(cur,config.stop_words), 'r', encoding="utf-8").readlines()]
        return stopkey


if __name__ == '__main__':
    loader = LoadData()
    print(loader.get_stop_words())
